package lion.spider;

import java.util.Date;

import org.apache.commons.codec.digest.DigestUtils;

import lion.crawl.parse.Page;
import lion.crawl.url.URLBase;
import lion.dev.lang.MapJ;
import lion.framework.core.db.DB;
import lion.framework.core.db.paging.SQLPagingFactory;
import lion.spider.model.CrawlerConfig;
import lion.spider.model.SpiderURLPage;

/**
 * 抓取页面URL库
 * 
 * @author lion
 * @email hl_0074@sina.com
 * @date 2016年5月6日
 */
public class SpiderURLBase implements URLBase {

	private volatile int id;

	private CrawlerConfig config;

	public SpiderURLBase(CrawlerConfig config) {
		this.config = config;

		try (DB db = new DB()) {
			this.id = db.queryInt("select max(id) from SpiderURLPage where crawled =true and configId=?", config.getId());
		}
	}

	@Override
	public void add(Page page) {

		SpiderURLPage urlPage = new SpiderURLPage();
		urlPage.setCrawled(page.isCrawled());
		if (page.isCrawled()) {
			urlPage.setCrawledtime(new Date());
		}
		urlPage.setMd5(DigestUtils.md5Hex(page.getContent()));
		urlPage.setConfigId(config.getId());
		urlPage.setUrl(page.getUrl());
		urlPage.setLevel(page.getLevel());
		try (DB db = new DB()) {
			urlPage.save(db);
		}
	}

	@Override
	public void add(URLBase other) {

		Page page;
		try (DB db = new DB()) {
			while ((page = other.get()) != null) {
				SpiderURLPage urlPage = new SpiderURLPage();
				urlPage.setCrawled(page.isCrawled());
				if (page.isCrawled()) {
					urlPage.setCrawledtime(new Date());
				}
				urlPage.setMd5(DigestUtils.md5Hex(page.getContent()));
				urlPage.setConfigId(config.getId());
				urlPage.setUrl(page.getUrl());

				urlPage.save(db);
			}
		}
	}

	@Override
	public boolean contains(Page page) {

		try (DB db = new DB()) {
			return db.queryTrue("select 1 from SpiderURLPage where url=? and configId=?", page.getUrl(), config.getId());
		}
	}

	@Override
	public void crawled(Page page) {

		try (DB db = new DB()) {
			db.update("update SpiderURLPage set crawled=1 , crawledtime=? where url=? and configId=?", new Date(), page.getUrl(), config.getId());
		}
	}

	@Override
	public Page get() {

		try (DB db = new DB()) {

			MapJ param = new MapJ();
			param.put("id", id);
			param.put("configId", config.getId());
			SpiderURLPage page = SpiderURLPage.load(SQLPagingFactory.paging(db.getType(), " where id>? and crawled=false and configId=?", 0, 1), param, db);

			if (page == null) { return null; }

			this.id = page.getId();

			Page result = new Page();
			result.setCrawled(false);
			result.setUrl(page.getUrl());
			result.setLevel(page.getLevel());
			return result;
		}
	}

	@Override
	public boolean hasNew() {

		try (DB db = new DB()) {
			return db.queryTrue("select 1 from SpiderURLPage where crawled=false and configId=?", config.getId());
		}
	}

	@Override
	public int size() {

		try (DB db = new DB()) {
			return SpiderURLPage.count("where configId=" + config.getId(), db);
		}
	}

}
